{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "24ae5547",
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "\n",
    "os.environ['CUDA_VISIBLE_DEVICES'] = '0'\n",
    "os.environ['TF_FORCE_GPU_ALLOW_GROWTH'] = 'true'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "3395870c",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/ubuntu/.local/lib/python3.8/site-packages/apex/pyprof/__init__.py:5: FutureWarning: pyprof will be removed by the end of June, 2022\n",
      "  warnings.warn(\"pyprof will be removed by the end of June, 2022\", FutureWarning)\n"
     ]
    }
   ],
   "source": [
    "from transformers import T5Config, TFT5Model, T5Model, load_tf_weights_in_t5, T5Tokenizer\n",
    "from transformers import T5ForConditionalGeneration, TFT5ForConditionalGeneration"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "06db4241",
   "metadata": {},
   "outputs": [],
   "source": [
    "!rm -rf temp2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "a541b0ab",
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "\n",
    "out = 'temp2'\n",
    "os.makedirs(out, exist_ok=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "48753c21",
   "metadata": {},
   "outputs": [],
   "source": [
    "config = T5Config.from_pretrained('malay-huggingface/t5-super-tiny-bahasa-cased')\n",
    "config.save_pretrained(out)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "50a47b1d",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "checkpoint\r\n",
      "events.out.tfevents.1659118087.husein-MS-7D31\r\n",
      "events.out.tfevents.1659118188.husein-MS-7D31\r\n",
      "events.out.tfevents.1659119840.husein-MS-7D31\r\n",
      "graph.pbtxt\r\n",
      "model.ckpt-1015000.data-00000-of-00002\r\n",
      "model.ckpt-1015000.data-00001-of-00002\r\n",
      "model.ckpt-1015000.index\r\n",
      "model.ckpt-1015000.meta\r\n",
      "model.ckpt-1047000.data-00000-of-00002\r\n",
      "model.ckpt-1047000.data-00001-of-00002\r\n",
      "model.ckpt-1047000.index\r\n",
      "model.ckpt-1047000.meta\r\n",
      "model.ckpt-1080000.data-00000-of-00002\r\n",
      "model.ckpt-1080000.data-00001-of-00002\r\n",
      "model.ckpt-1080000.index\r\n",
      "model.ckpt-1080000.meta\r\n",
      "model.ckpt-1112000.data-00000-of-00002\r\n",
      "model.ckpt-1112000.data-00001-of-00002\r\n",
      "model.ckpt-1112000.index\r\n",
      "model.ckpt-1112000.meta\r\n",
      "model.ckpt-1145000.data-00000-of-00002\r\n",
      "model.ckpt-1145000.data-00001-of-00002\r\n",
      "model.ckpt-1145000.index\r\n",
      "model.ckpt-1145000.meta\r\n",
      "model.ckpt-1177000.data-00000-of-00002\r\n",
      "model.ckpt-1177000.data-00001-of-00002\r\n",
      "model.ckpt-1177000.index\r\n",
      "model.ckpt-1177000.meta\r\n",
      "model.ckpt-1197000.data-00000-of-00002\r\n",
      "model.ckpt-1197000.data-00001-of-00002\r\n",
      "model.ckpt-1197000.index\r\n",
      "model.ckpt-1197000.meta\r\n",
      "model.ckpt-1198000.data-00000-of-00002\r\n",
      "model.ckpt-1198000.data-00001-of-00002\r\n",
      "model.ckpt-1198000.index\r\n",
      "model.ckpt-1198000.meta\r\n",
      "model.ckpt-1199000.data-00000-of-00002\r\n",
      "model.ckpt-1199000.data-00001-of-00002\r\n",
      "model.ckpt-1199000.index\r\n",
      "model.ckpt-1199000.meta\r\n",
      "model.ckpt-1200000.data-00000-of-00002\r\n",
      "model.ckpt-1200000.data-00001-of-00002\r\n",
      "model.ckpt-1200000.index\r\n",
      "model.ckpt-1200000.meta\r\n",
      "model.ckpt-1201000.data-00000-of-00002\r\n",
      "model.ckpt-1201000.data-00001-of-00002\r\n",
      "model.ckpt-1201000.index\r\n",
      "model.ckpt-1201000.meta\r\n",
      "model.ckpt-854000.data-00000-of-00002\r\n",
      "model.ckpt-854000.data-00001-of-00002\r\n",
      "model.ckpt-854000.index\r\n",
      "model.ckpt-854000.meta\r\n",
      "model.ckpt-886000.data-00000-of-00002\r\n",
      "model.ckpt-886000.data-00001-of-00002\r\n",
      "model.ckpt-886000.index\r\n",
      "model.ckpt-886000.meta\r\n",
      "model.ckpt-919000.data-00000-of-00002\r\n",
      "model.ckpt-919000.data-00001-of-00002\r\n",
      "model.ckpt-919000.index\r\n",
      "model.ckpt-919000.meta\r\n",
      "model.ckpt-951000.data-00000-of-00002\r\n",
      "model.ckpt-951000.data-00001-of-00002\r\n",
      "model.ckpt-951000.index\r\n",
      "model.ckpt-951000.meta\r\n",
      "model.ckpt-983000.data-00000-of-00002\r\n",
      "model.ckpt-983000.data-00001-of-00002\r\n",
      "model.ckpt-983000.index\r\n",
      "model.ckpt-983000.meta\r\n",
      "operative_config.gin\r\n"
     ]
    }
   ],
   "source": [
    "!ls t5-super-tiny-noisy-en-ms"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "f54394ad",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "T5Model(\n",
       "  (shared): Embedding(32128, 256)\n",
       "  (encoder): T5Stack(\n",
       "    (embed_tokens): Embedding(32128, 256)\n",
       "    (block): ModuleList(\n",
       "      (0): T5Block(\n",
       "        (layer): ModuleList(\n",
       "          (0): T5LayerSelfAttention(\n",
       "            (SelfAttention): T5Attention(\n",
       "              (q): Linear(in_features=256, out_features=384, bias=False)\n",
       "              (k): Linear(in_features=256, out_features=384, bias=False)\n",
       "              (v): Linear(in_features=256, out_features=384, bias=False)\n",
       "              (o): Linear(in_features=384, out_features=256, bias=False)\n",
       "              (relative_attention_bias): Embedding(32, 6)\n",
       "            )\n",
       "            (layer_norm): FusedRMSNorm(torch.Size([256]), eps=1e-06, elementwise_affine=True)\n",
       "            (dropout): Dropout(p=0.1, inplace=False)\n",
       "          )\n",
       "          (1): T5LayerFF(\n",
       "            (DenseReluDense): T5DenseActDense(\n",
       "              (wi): Linear(in_features=256, out_features=1024, bias=False)\n",
       "              (wo): Linear(in_features=1024, out_features=256, bias=False)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "              (act): ReLU()\n",
       "            )\n",
       "            (layer_norm): FusedRMSNorm(torch.Size([256]), eps=1e-06, elementwise_affine=True)\n",
       "            (dropout): Dropout(p=0.1, inplace=False)\n",
       "          )\n",
       "        )\n",
       "      )\n",
       "      (1): T5Block(\n",
       "        (layer): ModuleList(\n",
       "          (0): T5LayerSelfAttention(\n",
       "            (SelfAttention): T5Attention(\n",
       "              (q): Linear(in_features=256, out_features=384, bias=False)\n",
       "              (k): Linear(in_features=256, out_features=384, bias=False)\n",
       "              (v): Linear(in_features=256, out_features=384, bias=False)\n",
       "              (o): Linear(in_features=384, out_features=256, bias=False)\n",
       "            )\n",
       "            (layer_norm): FusedRMSNorm(torch.Size([256]), eps=1e-06, elementwise_affine=True)\n",
       "            (dropout): Dropout(p=0.1, inplace=False)\n",
       "          )\n",
       "          (1): T5LayerFF(\n",
       "            (DenseReluDense): T5DenseActDense(\n",
       "              (wi): Linear(in_features=256, out_features=1024, bias=False)\n",
       "              (wo): Linear(in_features=1024, out_features=256, bias=False)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "              (act): ReLU()\n",
       "            )\n",
       "            (layer_norm): FusedRMSNorm(torch.Size([256]), eps=1e-06, elementwise_affine=True)\n",
       "            (dropout): Dropout(p=0.1, inplace=False)\n",
       "          )\n",
       "        )\n",
       "      )\n",
       "    )\n",
       "    (final_layer_norm): FusedRMSNorm(torch.Size([256]), eps=1e-06, elementwise_affine=True)\n",
       "    (dropout): Dropout(p=0.1, inplace=False)\n",
       "  )\n",
       "  (decoder): T5Stack(\n",
       "    (embed_tokens): Embedding(32128, 256)\n",
       "    (block): ModuleList(\n",
       "      (0): T5Block(\n",
       "        (layer): ModuleList(\n",
       "          (0): T5LayerSelfAttention(\n",
       "            (SelfAttention): T5Attention(\n",
       "              (q): Linear(in_features=256, out_features=384, bias=False)\n",
       "              (k): Linear(in_features=256, out_features=384, bias=False)\n",
       "              (v): Linear(in_features=256, out_features=384, bias=False)\n",
       "              (o): Linear(in_features=384, out_features=256, bias=False)\n",
       "              (relative_attention_bias): Embedding(32, 6)\n",
       "            )\n",
       "            (layer_norm): FusedRMSNorm(torch.Size([256]), eps=1e-06, elementwise_affine=True)\n",
       "            (dropout): Dropout(p=0.1, inplace=False)\n",
       "          )\n",
       "          (1): T5LayerCrossAttention(\n",
       "            (EncDecAttention): T5Attention(\n",
       "              (q): Linear(in_features=256, out_features=384, bias=False)\n",
       "              (k): Linear(in_features=256, out_features=384, bias=False)\n",
       "              (v): Linear(in_features=256, out_features=384, bias=False)\n",
       "              (o): Linear(in_features=384, out_features=256, bias=False)\n",
       "            )\n",
       "            (layer_norm): FusedRMSNorm(torch.Size([256]), eps=1e-06, elementwise_affine=True)\n",
       "            (dropout): Dropout(p=0.1, inplace=False)\n",
       "          )\n",
       "          (2): T5LayerFF(\n",
       "            (DenseReluDense): T5DenseActDense(\n",
       "              (wi): Linear(in_features=256, out_features=1024, bias=False)\n",
       "              (wo): Linear(in_features=1024, out_features=256, bias=False)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "              (act): ReLU()\n",
       "            )\n",
       "            (layer_norm): FusedRMSNorm(torch.Size([256]), eps=1e-06, elementwise_affine=True)\n",
       "            (dropout): Dropout(p=0.1, inplace=False)\n",
       "          )\n",
       "        )\n",
       "      )\n",
       "      (1): T5Block(\n",
       "        (layer): ModuleList(\n",
       "          (0): T5LayerSelfAttention(\n",
       "            (SelfAttention): T5Attention(\n",
       "              (q): Linear(in_features=256, out_features=384, bias=False)\n",
       "              (k): Linear(in_features=256, out_features=384, bias=False)\n",
       "              (v): Linear(in_features=256, out_features=384, bias=False)\n",
       "              (o): Linear(in_features=384, out_features=256, bias=False)\n",
       "            )\n",
       "            (layer_norm): FusedRMSNorm(torch.Size([256]), eps=1e-06, elementwise_affine=True)\n",
       "            (dropout): Dropout(p=0.1, inplace=False)\n",
       "          )\n",
       "          (1): T5LayerCrossAttention(\n",
       "            (EncDecAttention): T5Attention(\n",
       "              (q): Linear(in_features=256, out_features=384, bias=False)\n",
       "              (k): Linear(in_features=256, out_features=384, bias=False)\n",
       "              (v): Linear(in_features=256, out_features=384, bias=False)\n",
       "              (o): Linear(in_features=384, out_features=256, bias=False)\n",
       "            )\n",
       "            (layer_norm): FusedRMSNorm(torch.Size([256]), eps=1e-06, elementwise_affine=True)\n",
       "            (dropout): Dropout(p=0.1, inplace=False)\n",
       "          )\n",
       "          (2): T5LayerFF(\n",
       "            (DenseReluDense): T5DenseActDense(\n",
       "              (wi): Linear(in_features=256, out_features=1024, bias=False)\n",
       "              (wo): Linear(in_features=1024, out_features=256, bias=False)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "              (act): ReLU()\n",
       "            )\n",
       "            (layer_norm): FusedRMSNorm(torch.Size([256]), eps=1e-06, elementwise_affine=True)\n",
       "            (dropout): Dropout(p=0.1, inplace=False)\n",
       "          )\n",
       "        )\n",
       "      )\n",
       "    )\n",
       "    (final_layer_norm): FusedRMSNorm(torch.Size([256]), eps=1e-06, elementwise_affine=True)\n",
       "    (dropout): Dropout(p=0.1, inplace=False)\n",
       "  )\n",
       ")"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model = T5Model(config)\n",
    "load_tf_weights_in_t5(model, config, 't5-super-tiny-noisy-en-ms/model.ckpt-1201000')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "86b9d748",
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "tokenizer = T5Tokenizer('sp10m.cased.ms-en.model', padding = True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "7e68dcc6",
   "metadata": {},
   "outputs": [],
   "source": [
    "model.save_pretrained(out)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "737e7c80",
   "metadata": {},
   "outputs": [],
   "source": [
    "model_gen = T5ForConditionalGeneration.from_pretrained(out)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "005b412a",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2022-08-01 15:04:16.736034: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:937] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero\n",
      "2022-08-01 15:04:16.740027: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:937] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero\n",
      "2022-08-01 15:04:16.740688: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:937] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero\n",
      "2022-08-01 15:04:16.741772: I tensorflow/core/platform/cpu_feature_guard.cc:142] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations:  AVX2 AVX512F FMA\n",
      "To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags.\n",
      "2022-08-01 15:04:16.742248: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:937] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero\n",
      "2022-08-01 15:04:16.742904: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:937] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero\n",
      "2022-08-01 15:04:16.743533: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:937] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero\n",
      "2022-08-01 15:04:21.137044: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:937] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero\n",
      "2022-08-01 15:04:21.137650: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:937] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero\n",
      "2022-08-01 15:04:21.138205: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:937] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero\n",
      "2022-08-01 15:04:21.138721: W tensorflow/core/common_runtime/gpu/gpu_bfc_allocator.cc:39] Overriding allow_growth setting because the TF_FORCE_GPU_ALLOW_GROWTH environment variable is set. Original config value was 0.\n",
      "2022-08-01 15:04:21.138738: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1510] Created device /job:localhost/replica:0/task:0/device:GPU:0 with 14990 MB memory:  -> device: 0, name: NVIDIA GeForce RTX 3090 Ti, pci bus id: 0000:01:00.0, compute capability: 8.6\n",
      "2022-08-01 15:04:21.148976: W tensorflow/python/util/util.cc:348] Sets are not currently considered sequences, but this may change in the future, so consider avoiding using them.\n",
      "2022-08-01 15:04:21.924642: I tensorflow/stream_executor/cuda/cuda_blas.cc:1760] TensorFloat-32 will be used for the matrix multiplication. This will only be logged once.\n",
      "Some weights of the PyTorch model were not used when initializing the TF 2.0 model TFT5ForConditionalGeneration: ['encoder.embed_tokens.weight', 'decoder.embed_tokens.weight']\n",
      "- This IS expected if you are initializing TFT5ForConditionalGeneration from a PyTorch model trained on another task or with another architecture (e.g. initializing a TFBertForSequenceClassification model from a BertForPreTraining model).\n",
      "- This IS NOT expected if you are initializing TFT5ForConditionalGeneration from a PyTorch model that you expect to be exactly identical (e.g. initializing a TFBertForSequenceClassification model from a BertForSequenceClassification model).\n",
      "All the weights of TFT5ForConditionalGeneration were initialized from the PyTorch model.\n",
      "If your task is similar to the task the model of the checkpoint was trained on, you can already use TFT5ForConditionalGeneration for predictions without further training.\n"
     ]
    }
   ],
   "source": [
    "model_tf = TFT5ForConditionalGeneration.from_pretrained(out, from_pt = True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "fc1a66fe",
   "metadata": {},
   "outputs": [],
   "source": [
    "with open('en-ms-right.test') as fopen:\n",
    "    right = fopen.read().split('\\n')\n",
    "    \n",
    "with open('en-ms-left.test') as fopen:\n",
    "    left = fopen.read().split('\\n')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "3124e42d",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "T5ForConditionalGeneration(\n",
       "  (shared): Embedding(32128, 256)\n",
       "  (encoder): T5Stack(\n",
       "    (embed_tokens): Embedding(32128, 256)\n",
       "    (block): ModuleList(\n",
       "      (0): T5Block(\n",
       "        (layer): ModuleList(\n",
       "          (0): T5LayerSelfAttention(\n",
       "            (SelfAttention): T5Attention(\n",
       "              (q): Linear(in_features=256, out_features=384, bias=False)\n",
       "              (k): Linear(in_features=256, out_features=384, bias=False)\n",
       "              (v): Linear(in_features=256, out_features=384, bias=False)\n",
       "              (o): Linear(in_features=384, out_features=256, bias=False)\n",
       "              (relative_attention_bias): Embedding(32, 6)\n",
       "            )\n",
       "            (layer_norm): FusedRMSNorm(torch.Size([256]), eps=1e-06, elementwise_affine=True)\n",
       "            (dropout): Dropout(p=0.1, inplace=False)\n",
       "          )\n",
       "          (1): T5LayerFF(\n",
       "            (DenseReluDense): T5DenseActDense(\n",
       "              (wi): Linear(in_features=256, out_features=1024, bias=False)\n",
       "              (wo): Linear(in_features=1024, out_features=256, bias=False)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "              (act): ReLU()\n",
       "            )\n",
       "            (layer_norm): FusedRMSNorm(torch.Size([256]), eps=1e-06, elementwise_affine=True)\n",
       "            (dropout): Dropout(p=0.1, inplace=False)\n",
       "          )\n",
       "        )\n",
       "      )\n",
       "      (1): T5Block(\n",
       "        (layer): ModuleList(\n",
       "          (0): T5LayerSelfAttention(\n",
       "            (SelfAttention): T5Attention(\n",
       "              (q): Linear(in_features=256, out_features=384, bias=False)\n",
       "              (k): Linear(in_features=256, out_features=384, bias=False)\n",
       "              (v): Linear(in_features=256, out_features=384, bias=False)\n",
       "              (o): Linear(in_features=384, out_features=256, bias=False)\n",
       "            )\n",
       "            (layer_norm): FusedRMSNorm(torch.Size([256]), eps=1e-06, elementwise_affine=True)\n",
       "            (dropout): Dropout(p=0.1, inplace=False)\n",
       "          )\n",
       "          (1): T5LayerFF(\n",
       "            (DenseReluDense): T5DenseActDense(\n",
       "              (wi): Linear(in_features=256, out_features=1024, bias=False)\n",
       "              (wo): Linear(in_features=1024, out_features=256, bias=False)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "              (act): ReLU()\n",
       "            )\n",
       "            (layer_norm): FusedRMSNorm(torch.Size([256]), eps=1e-06, elementwise_affine=True)\n",
       "            (dropout): Dropout(p=0.1, inplace=False)\n",
       "          )\n",
       "        )\n",
       "      )\n",
       "    )\n",
       "    (final_layer_norm): FusedRMSNorm(torch.Size([256]), eps=1e-06, elementwise_affine=True)\n",
       "    (dropout): Dropout(p=0.1, inplace=False)\n",
       "  )\n",
       "  (decoder): T5Stack(\n",
       "    (embed_tokens): Embedding(32128, 256)\n",
       "    (block): ModuleList(\n",
       "      (0): T5Block(\n",
       "        (layer): ModuleList(\n",
       "          (0): T5LayerSelfAttention(\n",
       "            (SelfAttention): T5Attention(\n",
       "              (q): Linear(in_features=256, out_features=384, bias=False)\n",
       "              (k): Linear(in_features=256, out_features=384, bias=False)\n",
       "              (v): Linear(in_features=256, out_features=384, bias=False)\n",
       "              (o): Linear(in_features=384, out_features=256, bias=False)\n",
       "              (relative_attention_bias): Embedding(32, 6)\n",
       "            )\n",
       "            (layer_norm): FusedRMSNorm(torch.Size([256]), eps=1e-06, elementwise_affine=True)\n",
       "            (dropout): Dropout(p=0.1, inplace=False)\n",
       "          )\n",
       "          (1): T5LayerCrossAttention(\n",
       "            (EncDecAttention): T5Attention(\n",
       "              (q): Linear(in_features=256, out_features=384, bias=False)\n",
       "              (k): Linear(in_features=256, out_features=384, bias=False)\n",
       "              (v): Linear(in_features=256, out_features=384, bias=False)\n",
       "              (o): Linear(in_features=384, out_features=256, bias=False)\n",
       "            )\n",
       "            (layer_norm): FusedRMSNorm(torch.Size([256]), eps=1e-06, elementwise_affine=True)\n",
       "            (dropout): Dropout(p=0.1, inplace=False)\n",
       "          )\n",
       "          (2): T5LayerFF(\n",
       "            (DenseReluDense): T5DenseActDense(\n",
       "              (wi): Linear(in_features=256, out_features=1024, bias=False)\n",
       "              (wo): Linear(in_features=1024, out_features=256, bias=False)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "              (act): ReLU()\n",
       "            )\n",
       "            (layer_norm): FusedRMSNorm(torch.Size([256]), eps=1e-06, elementwise_affine=True)\n",
       "            (dropout): Dropout(p=0.1, inplace=False)\n",
       "          )\n",
       "        )\n",
       "      )\n",
       "      (1): T5Block(\n",
       "        (layer): ModuleList(\n",
       "          (0): T5LayerSelfAttention(\n",
       "            (SelfAttention): T5Attention(\n",
       "              (q): Linear(in_features=256, out_features=384, bias=False)\n",
       "              (k): Linear(in_features=256, out_features=384, bias=False)\n",
       "              (v): Linear(in_features=256, out_features=384, bias=False)\n",
       "              (o): Linear(in_features=384, out_features=256, bias=False)\n",
       "            )\n",
       "            (layer_norm): FusedRMSNorm(torch.Size([256]), eps=1e-06, elementwise_affine=True)\n",
       "            (dropout): Dropout(p=0.1, inplace=False)\n",
       "          )\n",
       "          (1): T5LayerCrossAttention(\n",
       "            (EncDecAttention): T5Attention(\n",
       "              (q): Linear(in_features=256, out_features=384, bias=False)\n",
       "              (k): Linear(in_features=256, out_features=384, bias=False)\n",
       "              (v): Linear(in_features=256, out_features=384, bias=False)\n",
       "              (o): Linear(in_features=384, out_features=256, bias=False)\n",
       "            )\n",
       "            (layer_norm): FusedRMSNorm(torch.Size([256]), eps=1e-06, elementwise_affine=True)\n",
       "            (dropout): Dropout(p=0.1, inplace=False)\n",
       "          )\n",
       "          (2): T5LayerFF(\n",
       "            (DenseReluDense): T5DenseActDense(\n",
       "              (wi): Linear(in_features=256, out_features=1024, bias=False)\n",
       "              (wo): Linear(in_features=1024, out_features=256, bias=False)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "              (act): ReLU()\n",
       "            )\n",
       "            (layer_norm): FusedRMSNorm(torch.Size([256]), eps=1e-06, elementwise_affine=True)\n",
       "            (dropout): Dropout(p=0.1, inplace=False)\n",
       "          )\n",
       "        )\n",
       "      )\n",
       "    )\n",
       "    (final_layer_norm): FusedRMSNorm(torch.Size([256]), eps=1e-06, elementwise_affine=True)\n",
       "    (dropout): Dropout(p=0.1, inplace=False)\n",
       "  )\n",
       "  (lm_head): Linear(in_features=256, out_features=32128, bias=False)\n",
       ")"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model_gen.cuda()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "35abd80e",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|████████████████████████████████████████████████████████████████████████████████████| 69744/69744 [2:40:19<00:00,  7.25it/s]\n"
     ]
    }
   ],
   "source": [
    "from tqdm import tqdm\n",
    "\n",
    "batch_size = 1\n",
    "\n",
    "results = []\n",
    "for i in tqdm(range(0, len(left), batch_size)):\n",
    "    input_ids = [{'input_ids': tokenizer.encode(f'terjemah Inggeris ke Melayu: {s}', return_tensors = 'pt')[0]} for s in left[i:i + batch_size]]\n",
    "    padded = tokenizer.pad(input_ids, padding = 'longest')\n",
    "    outputs = model_gen.generate(padded['input_ids'].cuda(), attention_mask = padded['attention_mask'].cuda(), max_length = 1000)\n",
    "    for o in outputs:\n",
    "        results.append(tokenizer.decode(o, skip_special_tokens=True))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "9a28a3e4",
   "metadata": {},
   "outputs": [],
   "source": [
    "# from tqdm import tqdm\n",
    "\n",
    "# batch_size = 2\n",
    "\n",
    "# results = []\n",
    "# for i in tqdm(range(0, len(left[:1]), batch_size)):\n",
    "#     input_ids = [{'input_ids': tokenizer.encode(f'terjemah Inggeris ke Melayu: {s}', return_tensors = 'pt')[0]} for s in left[i:i + batch_size]]\n",
    "#     padded = tokenizer.pad(input_ids, padding = 'longest')\n",
    "#     outputs = model_gen.generate(**padded, max_length = 1000)\n",
    "#     for o in outputs:\n",
    "#         results.append(tokenizer.decode(o, skip_special_tokens=True))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "52c3b550",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'Anda tahu bagaimana kisahnya berjalan. Dua orang heteroseksual, hampir pasti orang kulit putih bergelut dalam hidup mereka. Mereka saling bertemu, berkumpul, berpisah, bersatu lagi dan mencari kebahagiaan dan penyelesaian yang benar di antara satu sama lain senjata. Akhirnya, mereka dapat memulakan hidup normal! Rom-com boleh menikmati eskapisme yang menyeronokkan, tetapi kisah yang terlalu sering diceritakan, yang terlalu banyak bergantung pada kepelbagaian, terlalu bergantung pada stereotaip jantina dan terlalu mementingkan penjualan kami jenama cinta yang tidak mungkin dapat hidup hingga: sebuah kajian di Heriot Watt University mendapati bahawa rom-com mempunyai kesan negatif terhadap hubungan, menjadikan kita mengejar standard cinta yang tidak dapat dicapai. Dalam proses menulis drama baru saya, Ross & Rachel, yang menghadapi mitos cinta moden dan terbuka di Festival Edinburgh Fringe Ogos ini, saya harus banyak memikirkan percintaan dalam fiksyen dari Romeo dan Juliet hingga Pride dan Prejudice sampai ke Notting Hill. Mengapa kita terus terus menceritakan kisah yang sama?'"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "results[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "eb550d6f",
   "metadata": {},
   "outputs": [],
   "source": [
    "from sacrebleu.metrics import BLEU, CHRF, TER\n",
    "\n",
    "bleu = BLEU()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "20530f31",
   "metadata": {},
   "outputs": [],
   "source": [
    "filtered_left, filtered_right = [], []\n",
    "for no, r in enumerate(results):\n",
    "    if len(r):\n",
    "        filtered_left.append(r)\n",
    "        filtered_right.append(right[no])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "e50b3fbd",
   "metadata": {},
   "outputs": [],
   "source": [
    "refs = [filtered_right]\n",
    "sys = filtered_left"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "361cc363",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'Anda tahu bagaimana kisahnya berjalan. Dua orang heteroseksual, hampir pasti orang kulit putih bergelut dalam hidup mereka. Mereka saling bertemu, berkumpul, berpisah, bersatu lagi dan mencari kebahagiaan dan penyelesaian yang benar di antara satu sama lain senjata. Akhirnya, mereka dapat memulakan hidup normal! Rom-com boleh menikmati eskapisme yang menyeronokkan, tetapi kisah yang terlalu sering diceritakan, yang terlalu banyak bergantung pada kepelbagaian, terlalu bergantung pada stereotaip jantina dan terlalu mementingkan penjualan kami jenama cinta yang tidak mungkin dapat hidup hingga: sebuah kajian di Heriot Watt University mendapati bahawa rom-com mempunyai kesan negatif terhadap hubungan, menjadikan kita mengejar standard cinta yang tidak dapat dicapai. Dalam proses menulis drama baru saya, Ross & Rachel, yang menghadapi mitos cinta moden dan terbuka di Festival Edinburgh Fringe Ogos ini, saya harus banyak memikirkan percintaan dalam fiksyen dari Romeo dan Juliet hingga Pride dan Prejudice sampai ke Notting Hill. Mengapa kita terus terus menceritakan kisah yang sama?'"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sys[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "id": "2ba0ce1c",
   "metadata": {},
   "outputs": [],
   "source": [
    "# for i in range(len(filtered_left)):\n",
    "#     print(filtered_left[i])\n",
    "#     print(filtered_right[i])\n",
    "#     print(right[i])\n",
    "#     print()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "id": "afa050d8",
   "metadata": {},
   "outputs": [],
   "source": [
    "# filtered_right"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "id": "ecc03af7",
   "metadata": {},
   "outputs": [],
   "source": [
    "# filtered_left"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "id": "e5d6f81d",
   "metadata": {},
   "outputs": [],
   "source": [
    "r = bleu.corpus_score(sys, refs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "id": "cf03103f",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'name': 'BLEU',\n",
       " 'score': 58.72114029430599,\n",
       " '_mean': -1.0,\n",
       " '_ci': -1.0,\n",
       " '_verbose': '80.4/64.0/53.0/44.7 (BP = 0.994 ratio = 0.994 hyp_len = 2614280 ref_len = 2630014)',\n",
       " 'bp': 0.9939995916897871,\n",
       " 'counts': [2102642, 1627808, 1311221, 1074639],\n",
       " 'totals': [2614280, 2544537, 2474822, 2405299],\n",
       " 'sys_len': 2614280,\n",
       " 'ref_len': 2630014,\n",
       " 'precisions': [80.42910476307053,\n",
       "  63.972659859141366,\n",
       "  52.982436716660835,\n",
       "  44.67797974389047],\n",
       " 'prec_str': '80.4/64.0/53.0/44.7',\n",
       " 'ratio': 0.9940175223401853}"
      ]
     },
     "execution_count": 27,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "r.__dict__"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "id": "88704b13",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/ubuntu/.local/lib/python3.8/site-packages/huggingface_hub/utils/_deprecation.py:39: FutureWarning: Pass token='t5-super-tiny-finetuned-noisy-en-ms' as keyword args. From version 0.7 passing these as positional arguments will result in an error\n",
      "  warnings.warn(\n",
      "/home/ubuntu/.local/lib/python3.8/site-packages/huggingface_hub/hf_api.py:79: FutureWarning: `name` and `organization` input arguments are deprecated and will be removed in v0.7. Pass `repo_id` instead.\n",
      "  warnings.warn(\n",
      "/home/ubuntu/.local/lib/python3.8/site-packages/huggingface_hub/hf_api.py:596: FutureWarning: `create_repo` now takes `token` as an optional positional argument. Be sure to adapt your code!\n",
      "  warnings.warn(\n",
      "Cloning https://huggingface.co/mesolitica/t5-super-tiny-finetuned-noisy-en-ms into local empty directory.\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "7dface9b887540a0a30139fd2fff1611",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Download file events.out.tfevents.1659119840.husein-MS-7D31:   0%|          | 15.6k/56.4M [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "eea22ecf3d6647c4a3e271ff33d3e420",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Download file pytorch_model.bin:   0%|          | 1.82k/48.4M [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "f67c4eff94a042fdb41265a07e05875c",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Download file tf_model.h5:   0%|          | 1.81k/48.5M [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "3f4edb9f23cf4694bd80eb412d146299",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Download file events.out.tfevents.1659118188.husein-MS-7D31:   0%|          | 4.24k/17.9M [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "ca941aa71da246bab316bf6294231816",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Download file spiece.model:   1%|1         | 8.00k/784k [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "063d5d06d6d54a9ea51bd5a3c344cf53",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Download file events.out.tfevents.1659118087.husein-MS-7D31:   0%|          | 1.58k/17.4M [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "ee373efe713e4bd3a95f33874e760e6d",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Clean file spiece.model:   0%|          | 1.00k/784k [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "7a03360629e64c11a7239a90a307686e",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Clean file events.out.tfevents.1659118087.husein-MS-7D31:   0%|          | 1.00k/17.4M [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "f66f8d2c21dd4281bd6741ac56bad0e1",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Clean file events.out.tfevents.1659118188.husein-MS-7D31:   0%|          | 1.00k/17.9M [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "6385cea846fd432a9f7f16d577c7d743",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Clean file events.out.tfevents.1659119840.husein-MS-7D31:   0%|          | 1.00k/56.4M [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "0ac55a821cab4e57a49c2fe58afd56af",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Clean file pytorch_model.bin:   0%|          | 1.00k/48.4M [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "e4441a8a9262418eb8008e99d95d8794",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Clean file tf_model.h5:   0%|          | 1.00k/48.5M [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "tokenizer.push_to_hub('t5-super-tiny-finetuned-noisy-en-ms', organization='mesolitica')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "id": "97afc0d5",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "9c04a3d0617b4b4cb7ada5e88d84d9e1",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Upload file pytorch_model.bin:   0%|          | 4.00k/48.4M [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "To https://huggingface.co/mesolitica/t5-super-tiny-finetuned-noisy-en-ms\n",
      "   1c432db..af5a567  main -> main\n",
      "\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "'https://huggingface.co/mesolitica/t5-super-tiny-finetuned-noisy-en-ms/commit/af5a567b8973f9345bef6ab32d8fe66863f3cae6'"
      ]
     },
     "execution_count": 29,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model_gen.push_to_hub('t5-super-tiny-finetuned-noisy-en-ms', organization='mesolitica')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "id": "db817c07",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "616db4760bd243969ee8ea412993f63e",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Upload file tf_model.h5:   0%|          | 4.00k/48.5M [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "To https://huggingface.co/mesolitica/t5-super-tiny-finetuned-noisy-en-ms\n",
      "   af5a567..786766e  main -> main\n",
      "\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "'https://huggingface.co/mesolitica/t5-super-tiny-finetuned-noisy-en-ms/commit/786766e3cd6dd1ef6f5ffdc6e42566eeadc0eb65'"
      ]
     },
     "execution_count": 30,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model_tf.push_to_hub('t5-super-tiny-finetuned-noisy-en-ms', organization='mesolitica')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "id": "2819f991",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Already up to date.\r\n"
     ]
    }
   ],
   "source": [
    "!cd t5-super-tiny-finetuned-noisy-en-ms && git pull"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "id": "df9b8117",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "On branch main\r\n",
      "Your branch is up to date with 'origin/main'.\r\n",
      "\r\n",
      "nothing to commit, working tree clean\r\n"
     ]
    }
   ],
   "source": [
    "!cp t5-super-tiny-noisy-en-ms/*.tfevents.* t5-super-tiny-finetuned-noisy-en-ms\n",
    "!cd t5-super-tiny-finetuned-noisy-en-ms && git add . && git commit -m 'add tensorboard' && git push"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "503c5687",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.10"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}
